#抓取最大资源地址 大目录地址
from bs4 import BeautifulSoup
import requests
import collections
import pymssql
import re
import datetime
import queue
import time
import threading
import sys

class zuiDaUrl(object):
    _surlcode=[5,6,7,8,9,10,11,22,12,13,14,15,19,20,21,3,4]    
    _server=''
    _user=''
    _password=''
    _dbname=''
    _endtime=None
    _showurl=False #打印 请求地址信息
    _lst=queue.Queue()#队列
    _reqipport='' #代理IP:端口
    _reqid=0
    _useProxy=False #启用代理
    _showDb=False #显示数据库信息
    _waitDeal=queue.Queue()#待处理的Html队列
    _lockr = threading.Lock()  # 全局锁
    _lockw = threading.Lock()  # 全局锁
    def __init__(self,server:str,user:str,pwd:str,dbname:str,endtime:datetime.datetime,showurl=False,useProxy=False,showDb=False):
        self._server=server
        self._user=user
        self._password=pwd
        self._dbname=dbname
        self._endtime=endtime
        self._showurl=showurl
        self._useProxy=useProxy
        self._showDb=showDb
    def __recordToMssql(self,rurl,vname,vtype,vcode,utime):
        rows=self.__readMssqlData('select * from ZuiDaSource where ReqUrl=%s',(rurl))
        if len(rows)==0:
            self.__writeToMssql('insert into ZuiDaSource(ReqUrl,VideoName,VideoType,TypeCode,UpdateTime)values(%s,%s,%s,%d,%s)',[(rurl,vname,vtype,vcode,utime)])
        else:
            self.__writeToMssql('update ZuiDaSource set UpdateTime=%s,PullTime=getdate(),isDeal=0,VideoName=%s where ReqUrl=%s and typecode not in (5,6,7,8,9,10,11,22) ', [
                                (utime, vname, rurl)])
            
    # 保存详情
    def __saveZuiDetail(self,pid,mcheng,bming,dyan,zyan,lxing,dqu,yyan,sying,pchang,gxing,jqing,pfen,zping,pfcishu,ialt,isrc,bzhu):
        rows=self.__readMssqlData('select * from ZuiDaDetail where ParentId=%d',(pid)) #执行SQL 
        if len(rows)==0:
            self.__writeToMssql('insert into [dbo].[ZuiDaDetail]([ParentId],[MingChen],[BieMing],[DaoYan],[ZhuYan],[LeiXing],[DiQu],[YuYan],[ShangYing],[PianChang],[GengXing],[Juqing],[PingFen],[ZongPin],[PinFen],[ImgUrl],[ImgAlt],[Remark])values(%d,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
            [(pid,mcheng,bming,dyan,zyan,lxing,dqu,yyan,sying,pchang,gxing,jqing,pfen,zping,pfcishu,isrc,ialt,bzhu)])
        else:
            self.__writeToMssql('update ZuiDaDetail set Remark=%s  where ParentId=%d',[(bzhu,pid)]) 
            
    #保存 mu38地址
    def __saveMu38Url(self,pid,title,urls):
        rows=self.__readMssqlData('select * from [ZuiDaMu38] where ParentId=%d and PlayUrl=%s',(pid,urls))
        if len(rows)==0:
            self.__writeToMssql('insert into [dbo].[ZuiDaMu38]([ParentId],[Title],[PlayUrl])values(%d,%s,%s)',[(pid,title,urls)])
    

    #保存 直接播放地址 不需要Dplayer.js
    def __savePlayUrl(self,pid,title,urls):
        rows=self.__readMssqlData('select * from [ZuiDaPlay] where ParentId=%d and PlayUrl=%s',(pid,urls))
        if len(rows)==0:
            self.__writeToMssql('insert into [dbo].[ZuiDaPlay]([ParentId],[Title],[PlayUrl])values(%d,%s,%s)',[(pid,title,urls)])
    
    #保存 Mp4下载地址
    def __saveDownUrl(self,pid,title,urls):
        rows=self.__readMssqlData('select * from [ZuiDaDown] where ParentId=%d and DownUrl=%s',(pid,urls))
        if len(rows)==0:
            self.__writeToMssql('insert into [dbo].[ZuiDaDown]([ParentId],[Title],[DownUrl])values(%d,%s,%s)',[(pid,title,urls)])
    
    # 加载代理IP
    def __loadProxyIp(self):
        rows=self.__readMssqlData('select Id,IpAddress,Port from [DaiLiIpAddress] where [IsValid]=%d ',(1))
        for r in rows:
            self._lst.put({'id':r[0],'ip':r[1],'port':str(r[2])})
        ip=self._lst.get()    
        self._reqipport=ip['ip']+':'+ip['port']
        self._reqid=ip['id']
        
    # 切换代理IP 
    def __changeProxyIp(self):
        self.__writeToMssql('update [dbo].[DaiLiIpAddress] set IsValid=0 where Id=%d',[(self._reqid)])
        ip=self._lst.get()
        self._reqipport=ip['ip']+':'+ip['port']
        self._reqid=ip['id']
    
    # 变更主表状态
    def __changeState(self,pid):
        self.__writeToMssql('update [dbo].[ZuiDaSource] set IsDeal=1 where Id=%d',[(pid)])        
    
    # 扫描Html
    def __scanUrl(self,htmlurl:str,tpcode:int):
        try:
            proxy={'http':self._reqipport}
            if tpcode==1:
                proxy={'https':self._reqipport}
            if self._showurl:
                print(htmlurl)
                if self._useProxy:
                    print(proxy)
            req= requests.get(url=htmlurl,proxies=proxy) if self._useProxy else requests.get(url=htmlurl)
            html=req.text
            bf=BeautifulSoup(html,'lxml')
            f=bf.find_all('div',class_='xing_vb')
            arrs=f[0].find_all('li')
            res=re.search('id-\d+',htmlurl).group(0)
            ctype=re.search('\d+',res).group(0)
            for l in arrs:    
                spans=l.find_all('span')
                sns=l.find_all('span',class_='xing_vb3')
                if len(spans)>1 and len(sns)==0:
                    pnm=l.find_all('span',class_='xing_vb4')#影片名称
                    pna=l.find_all('a')#影片地址
                    pnt=l.find_all('span',class_='xing_vb5')#影片类型
                    pnd=l.find_all('span',class_='xing_vb6')#更新日期
                    utiem=datetime.datetime.strptime(pnd[0].text,'%Y-%m-%d')
                    if self._endtime>utiem:
                        break
                    else:
                        self.__recordToMssql('http://www.zuidazy3.net'+pna[0]['href'],pnm[0].text,pnt[0].text,ctype,pnd[0].text)
                elif len(sns)==0:
                    atag=l.find_all('a',class_='pagelink_a')
                    # print(atag)
                    if len(atag)>1 and atag[len(atag)-2].text=='下一页':                
                        self.__scanUrl('http://www.zuidazy3.net'+atag[len(atag)-2]['href'],0)
        except Exception as e:
            print('请求出错')
            print(e)
            if self._useProxy:
                print('代理IP地址无效了')
                if tpcode==0:
                    self.__scanUrl(htmlurl,1)
                else:
                    self.__changeProxyIp() 
                    self.__scanUrl(htmlurl,0)
    
    #抓取Html内容
    def __DownHtmlContext(self,pid,htmlurl,tpcode):
        try:
            proxy={'http':self._reqipport}
            if tpcode==1:
                proxy={'https':self._reqipport}
            if self._showurl:
                print(htmlurl)
                if self._useProxy:
                    print(proxy)
            req= requests.get(url=htmlurl,proxies=proxy) if self._useProxy else requests.get(url=htmlurl)
            html=req.text
            bf=BeautifulSoup(html,'lxml')
            f=bf.find_all('div',class_='warp')
            if len(f)>0:
                imgs=f[0].find_all('img',class_='lazy')
                ialt=imgs[0]['alt'] if len(imgs)>0 else ''
                isrc=imgs[0]['src'] if len(imgs)>0 else ''   
                vodh=f[0].find_all('div',class_='vodh')
                mc=vodh[0].find_all('h2')[0].text #影片名称
                bzsm=vodh[0].find_all('span')[0].text #备注
                pfen=vodh[0].find_all('label')[0].text #评分   
                vodinfobox=f[0].find_all('div',class_='vodinfobox')
                lic=vodinfobox[0].find_all('li')
                bming=lic[0].find_all('span')[0].text
                dyan=lic[1].find_all('span')[0].text
                zyan=lic[2].find_all('span')[0].text
                lxing=lic[3].find_all('span')[0].text
                dqu=lic[4].find_all('span')[0].text
                yyan=lic[5].find_all('span')[0].text
                sying=lic[6].find_all('span')[0].text
                pchang=lic[7].find_all('span')[0].text
                gxin=lic[8].find_all('span')[0].text
                zpin=lic[11].find_all('span')[0].text
                pfencs=lic[12].find_all('span')[0].text
                jqing=lic[13].find_all('span',class_='more')[0].text
                # print('图片alt：'+ialt)
                # print('图片src：'+isrc)
                # print('名称：'+mc)
                # print('备注：'+bzsm)
                # print('评分：'+pfen)
                # print('别名：'+bming)
                # print('导演：'+dyan)
                # print('主演：'+zyan)
                # print('类型：'+lxing)
                # print('地区：'+dqu)
                # print('语言：'+yyan)
                # print('上映：'+sying)
                # print('片长：'+pchang)
                # print('更新：'+gxin)
                # print('总评分数：'+zpin)
                # print('评分次数：'+pfen)
                # print('剧情：'+jqing)
                self.__saveZuiDetail(pid,mc,bming,dyan,zyan,lxing,dqu,yyan,sying,pchang,gxin,jqing,pfen,zpin,pfencs,ialt,isrc,bzsm)
                p1=f[0].find_all('div',id='play_1')
                p2=f[0].find_all('div',id='play_2')
                d1=f[0].find_all('div',id='down_1')
                if len(p2)>0:
                    p2list = p2[0].find_all('li')
                    if len(p2list)>0:
                        for l in p2list:
                            if '$' in l.text:
                                arrs=l.text.split('$',1)
                                self.__savePlayUrl(pid,arrs[0],arrs[1])
                    # print(htmlurl+'p2')
                if len(p1)>0:
                    p1list = p1[0].find_all('li')
                    if len(p1list)>0:
                        for l in p1list:
                            if '$' in l.text:
                                arrs=l.text.split('$',1)
                                self.__saveMu38Url(pid,arrs[0],arrs[1])
                    # print(htmlurl+'p1')
                if len(d1)>0:
                    d1list = d1[0].find_all('li')
                    if len(d1list)>0:
                        for l in d1list:
                            if '$' in l.text:
                                arrs=l.text.split('$',1)
                                self.__saveDownUrl(pid,arrs[0],arrs[1])
                    # print(htmlurl+'d1')
                # if isrc!='':
                #     download_img(pid,isrc)
                self.__changeState(pid)
        except Exception as e:
            print(htmlurl)
            print('请求出错')
            print(sys.exc_info())
            print(e)
            if self._useProxy:
                print('代理IP地址无效了')
                if tpcode==0:
                    self.__DownHtmlContext(pid,htmlurl,1)
                else:
                    self.__changeProxyIp() 
                    self.__DownHtmlContext(pid,htmlurl,0)
    # 读取数据
    def __readMssqlData(self,sql:str,params:tuple):
        self._lockr.acquire()  # 修改数据前加锁
        conn=pymssql.connect(self._server,self._user,self._password,self._dbname) #连接数据库 (服务器地址,登录数据库账号,登录数据库账号密码,指定数据库)
        cursor=conn.cursor() #获取光标
        try:
            cursor.execute(sql,params) #执行SQL        
            rows=cursor.fetchall() #获取所有数据
            conn.commit()
            self._lockr.release()  # 修改数据前解锁
            return rows            
        except Exception as e: 
            self._lockr.release()  # 修改数据前解锁
            print(e)   #输出异常
            return []
        finally:
            conn.close()  # 数据库操作完成后要关闭连接
    
    # 保存数据  
    def __writeToMssql(self,sql:str,params:list):  
        self._lockw.acquire()  # 修改数据前加锁
        conn=pymssql.connect(self._server,self._user,self._password,self._dbname) #连接数据库 (服务器地址,登录数据库账号,登录数据库账号密码,指定数据库)
        cursor=conn.cursor() #获取光标
        try:
            cursor.executemany(sql,params)
            conn.commit()
            self._lockw.release()  # 修改数据前解锁
        except Exception as e:
            self._lockw.release()  # 修改数据前解锁
            print('保存出现异常')
            print(e)   #输出异常
        finally:
            conn.close()  # 数据库操作完成后要关闭连接
    
    # 程序入库
    def startScan(self):
        if self._showDb:
            print(self._server)
            print(self._user)
            print(self._dbname)
            print(self._endtime) 
        #数据抓取    程序入口
        for s in self._surlcode:
            self.__scanUrl('http://www.zuidazy3.net/?m=vod-type-id-{0}.html'.format(s), 0)
    
    # 下载Html 数据
    def LoadWillDown(self,passtime):
        rows = self.__readMssqlData(
            'select Id,ReqUrl,TypeCode from [ZuiDaSource] where IsDeal=0  ', ())
        print('本次需要采集【'+str(len(rows))+'】个 HTML页面')
        tlist = []
        #数据抓取    程序入口
        for s in self._surlcode:
            lst = [p for p in rows if p[2] == s]
            if len(lst) > 0:
                t = threading.Thread(
                    target=self.__downHtmlData, args=(lst, passtime))
                t.start()
                tlist.append(t)
                time.sleep(2)
        for t in tlist:
            t.join()
    # 自动采集
    def AutoCollect(self):
        self.startScan()
        self.LoadWillDown(3)
    def __downHtmlData(self,rows,sleeptime):
        for r in rows:
            self.__DownHtmlContext(r[0], r[1], 0)
            time.sleep(sleeptime)
    # _waitDeal
    # def __useThreading(self):
        
##########################电影##############################
#http://www.zuidazy3.net/?m=vod-type-id-5.html   动作片
#http://www.zuidazy3.net/?m=vod-type-id-6.html   喜剧片
#http://www.zuidazy3.net/?m=vod-type-id-7.html   爱情片
#http://www.zuidazy3.net/?m=vod-type-id-8.html   科幻片
#http://www.zuidazy3.net/?m=vod-type-id-9.html   恐怖片
#http://www.zuidazy3.net/?m=vod-type-id-10.html   剧情片
#http://www.zuidazy3.net/?m=vod-type-id-11.html   战争片
#http://www.zuidazy3.net/?m=vod-type-id-22.html   记录片
##########################电视剧##############################
#http://www.zuidazy3.net/?m=vod-type-id-12.html   国产剧
#http://www.zuidazy3.net/?m=vod-type-id-13.html   香港剧
#http://www.zuidazy3.net/?m=vod-type-id-14.html   韩国剧
#http://www.zuidazy3.net/?m=vod-type-id-15.html   欧美剧
#http://www.zuidazy3.net/?m=vod-type-id-19.html   台湾剧
#http://www.zuidazy3.net/?m=vod-type-id-20.html   日本剧
#http://www.zuidazy3.net/?m=vod-type-id-21.html   海外剧
###########################综艺###################################
#http://www.zuidazy3.net/?m=vod-type-id-3.html 综艺
###########################动漫###################################
#http://www.zuidazy3.net/?m=vod-type-id-4.html
